package com.saibo.spider.gather;

import com.saibo.spider.commons.SubmitRedisCommons;
import com.saibo.spider.pojo.SinaTopicInfo;
import com.saibo.spider.utils.*;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.util.*;

/**
 * 请求微博话题一小时
 * 
 * @author TerransForce
 *
 */
public class MouldWeiboTopicOneHour {

	Logger log = Logger.getLogger(MouldWeiboTopicOneHour.class);

	String nowUrl = "http://d.weibo.com/100803";;
	String nextPage = "";
	long cdate;

	public static void main(String[] args) throws InterruptedException {
		MouldWeiboTopicOneHour a = new MouldWeiboTopicOneHour();
		a.aa();
	}

	public void aa() throws InterruptedException {
		cdate = System.currentTimeMillis();
		String html = connect("http://d.weibo.com/100803", null);
		nextPage = parse(html);
		Thread.sleep(20 * 1000);
		while (true) {
			mission();
			if(null == nextPage){
				break;
			}
			Thread.sleep(20 * 1000);
		}
	}

	public void mission() {
		String html = connect(nextPage, nowUrl);
		nowUrl = nextPage;
		nextPage = parse(html);
		if (("1").equals(nextPage)) {
			for (int i = 0; i < 3; i++) {
				html = connect(nextPage, nowUrl);
				nowUrl = nextPage;
				nextPage = parse(html);
				if (!"1".equals(nextPage))
					break;
			}
		}
	}

	public String connect(String url, String refferer) {
		for(int i = 0 ; i < 3 ;i++){
			CloseableHttpResponse httpResponse = HttpRequestUtil.getRequestMethod(url, requestHeader(refferer));
			if (null == httpResponse) {
				System.out.println("请求失败，响应空");
				continue;
			}
			String html = HttpResponseUtil.getHtml(httpResponse);
			return html;
		}
		return null;
	}

	public String parse(String html) {
		if (StringUtil.isNullOrEmpty(html))
			return "1";
		List<SinaTopicInfo> topicList = new ArrayList<SinaTopicInfo>();
		Document document = Jsoup.parse(html);
		Elements scriptEle = document.select("script");
		List<String> jsonList = SinaBlogScriptUtil.replaceScript(scriptEle);
		for (String string : jsonList) {
			Map<String, Object> mso = StringUtil.jsonToMap(string);
			if (mso.get("ns").toString().equals("pl.content.miniTab.index")) {
				if (mso.get("domid").toString().startsWith("Pl_Discover_Pt")) {
					if (StringUtil.isNullOrEmpty(mso.get("html"))) {
						return "1";
					}
					Document document_page = Jsoup.parse(mso.get("html").toString());
					Elements element_WB_cardwrap = document_page
							.select("div.WB_cardwrap>div.DSC_pictext_b>div.WB_innerwrap>div.m_wrap>ul.pt_ul>li");
					for (Element element : element_WB_cardwrap) {
						SinaTopicInfo topic = new SinaTopicInfo();
						Element huatiElement = element.select("li>div.pic_txt>div.pic_box>a").first();
						String huati_url = huatiElement.attr("href");
						Element huatiImgElement = huatiElement.select("a>img").first();
						String huati_img_url = huatiImgElement.attr("src");
						String huati_name = huatiImgElement.attr("alt");
						Element rank_ele = element.select("li>div.pic_txt>div.info_box>div.text_box>div.title>span")
								.first();
						String rank = StringUtil.element2text(rank_ele);
						Element classifyEle = element
								.select("li>div.pic_txt>div.info_box>div.text_box>div.title>a.W_btn_tag").first();
						String classify = StringUtil.element2text(classifyEle).replaceAll(" ", "");

						if (StringUtil.isNullOrEmpty(classify))
							classify = "未分类";

						Element infoBox_Ele = element.select("li>div.pic_txt>div.info_box").first();
						Element subtEle = infoBox_Ele.select("div.info_box>div.text_box>div.subtitle").first();
						String sub = StringUtil.element2text(subtEle);
						Element sub_boxEle = infoBox_Ele
								.select("div.info_box>div.subinfo>div.sub_box>span>span>span.number").first();
						String read_num = StringUtil.element2text(sub_boxEle);
						Element authorEle = infoBox_Ele.select("div.info_box>div.subinfo>div.sub_box>span>span>a.tlink")
								.first();
						String author_nickname = StringUtil.element2text(authorEle);
						String author_url = authorEle.attr("href");
						System.out.println(getRank(rank) + "\t" + huati_name + "\t" + classify + "\t" + sub + "\t"
								+ numberFormat(read_num) + "\t" + author_nickname + "\t" + huati_url + "\t"
								+ huati_img_url + "\t" + author_url);

						topic.setAuthor_nickname(author_nickname);
						topic.setAuthor_rowkey(MD5Util.GET16BITMD5(filterUrl(author_url)));
						topic.setAuthor_url(filterUrl(author_url));

						Date date = new Date(cdate);
						topic.setCrawler_date(date);

						topic.setBrief(sub);
						topic.setClassify(classify);
						topic.setRank(getRank(rank));
						topic.setRead_count(numberFormat(read_num) + "");
						topic.setTitle(huati_name);
						topic.setTopic_img(huati_img_url);
						topic.setUrl(huati_url);
						topic.setRowkey(MD5Util.GET16BITMD5(huati_url));
						topicList.add(topic);

					}
					SubmitRedisCommons.submitRedisData("sina_topic_info", StringUtil.objectToJson(topicList));
					Elements element_nextPage = document_page
							.select("div.WB_cardwrap>div.DSC_pictext_b>div.WB_cardpage>div.W_pages>a.next");
					if (null == element_nextPage)
						return null;
					return "http://d.weibo.com/" + element_nextPage.attr("href");
				}
			}
		}
		return null;
	}

	/**
	 * 过滤URL
	 */
	public String filterUrl(String url) {
		if (url.contains("?")) {
			String[] surl = url.split("\\?");
			return surl[0];
		}
		return url;
	}

	public int getRank(String rank) {
		if ("TOP1".equals(rank)) {
			return 1;
		} else if ("TOP2".equals(rank)) {
			return 2;
		} else if ("TOP3".equals(rank)) {
			return 3;
		}else if("推荐".equals(rank)){
			return 0;
		}else {
			return Integer.parseInt(rank);
		}
	}

	public Long numberFormat(String num) {
		try {
			Long readCount = new Long(1);
			if (num.contains("亿")) {
				readCount = readCount * 100000000;
			} else if (num.contains("万")) {
				readCount = readCount * 10000;
			}
			num = num.replaceAll("万", "");
			num = num.replaceAll("亿", "");
			num = num.replaceAll(" ", "");
			num = num.replaceAll("	", "");
			num = num.trim();

			double cacheRead = Double.parseDouble(num);

			Double f = new Double(cacheRead * readCount);
			return f.longValue();
		} catch (Exception e) {
			e.printStackTrace();
			log.info("[微博话题-数字类型转换出错,原数据:" + num + "]");
			return 0L;
		}
	}

	/**
	 * 请求头
	 * 
	 * @return
	 */
	public Map<String, String> requestHeader(String referer) {
		Map<String, String> requestHeader = new HashMap<String, String>();
		requestHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		requestHeader.put("Accept-Encoding", "gzip, deflate, sdch");
		requestHeader.put("Accept-Language", "zh-CN,zh;q=0.8");
		requestHeader.put("Connection", "keep-alive");
		requestHeader.put("Cookie", FileUtil.txt2String(new File("C:/sinalogincookie.txt")));
		requestHeader.put("Host", "d.weibo.com");
		if (!StringUtil.isNullOrEmpty(referer))
			requestHeader.put("Referer", referer);
		requestHeader.put("Upgrade-Insecure-Requests", "1");
		requestHeader.put("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");
		return requestHeader;
	}
}
